#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = "Mark Perry"
__license__ = "GPLv2"

# TODO: definition : stripping...

"""
strip unwanted wikipedia markup
"""
import re, string, os
from subprocess import *

# variables
cat_re = re.compile(r'\[\[Category:.*', re.IGNORECASE | re.DOTALL)
lang_re = re.compile(r"\{\{lang (?:[^\}\|]*\|)* ([^\}\{\n\r]*) \}\}", re.VERBOSE)
strip_re = re.compile(r"""
      <ref>.*?</ref>
    | <ref\s.*?(/>|</ref>)        # references
    | <!--.*?-->                  # comments
    | <gallery>.*?</gallery>      # image gallery
    | <[^<]*?>                    # tags
    | [\*#]+[\s]*\n               # empty list item
    | __[\w]*?__                  # magic words
    | -{3,}                       # dashes
    | [<>{}]                      # left over character
""", re.VERBOSE | re.DOTALL | re.MULTILINE | re.IGNORECASE)
extra_re = re.compile(r"[\[\]\*#{}]|''+|==+|^[\s]*[:;]", re.MULTILINE)

# link replacing
int_link_re = re.compile(r'\[\[  (?: ([^\]\[<>]*?) \| )?  ([^\[\n\r<>]+?)  \]\]', re.VERBOSE)
def int_link_replace(match):
    if match.group(1) == None:
        dest = match.group(2).replace(' ', '_')
        dest = extra_re.sub('', dest)
        return '<a href="/wiki/%s">%s</a>' % (dest, match.group(2))
    else:
        dest = extra_re.sub('', match.group(1))
        return '<a href="/wiki/%s" title="%s">%s</a>' % (dest.replace(' ', '_'), dest.title(), match.group(2))

ext_link_re = re.compile(r'\[  (?: ([^\]\[<>]*?) \s )?  ([^\[\n\r<>]+?)  \]', re.VERBOSE)
def ext_link_replace(match):
    if match.group(1) == None:
        dest = extra_re.sub('', match.group(2))
        return '<a href="%s">%s</a>' % (dest, match.group(2))
    else:
        dest = extra_re.sub('', match.group(1))
        return '<a href="%s">%s</a>' % (dest, match.group(2))


# main functions
def strip(text):
    """ strip all formatting """
    text = strip_blocks(text)
    
    # replace links
    text = int_link_re.sub(r'\2',text)
    text = ext_link_re.sub(r'\2',text)
    
    # strip any unwanted characters
    text = extra_re.sub('',text)
    
    return text

def strip_blocks(text):
    """ Strip blocks of unwanted formatting """
    # fix html codes
    text = text.replace('&amp;','&')
    
    # strip everything after categories
    text = cat_re.sub('',text)
    
    # fix language tables
    text = lang_re.sub(r'\1',text)
    
    # remove tables
    text = remove('{{','}}',text)
    text = remove('{|','|}',text)
    text = remove('{','}',text)
    
    # remove images
    text = remove('[[',']]',text, init = r'\[\[image:')
    
    # replace html tags
    text = text.replace('&lt;','<')
    text = text.replace('&gt;','>')

    # remove misc
    text = strip_re.sub('',text)
    
    # change back left over tags
    text = text.replace('<', '&lt;')
    text = text.replace('>', '&gt;')
    
    return text

""" Remove text between start and end 
    with init being a different regex start """
def remove(start, end, text, init = None):
    
    def find_start():    
        if init == None:return text.find(start)
        s = re.search(init, text, re.IGNORECASE)
        if s == None: return -1
        return s.start()
        
    a = find_start()
    while a != -1:
        b = c = a
        while b <= c and b != -1:
            b = text.find(start, b+len(start))
            c = max(c,text.find(end, c+len(end)))
        text = text[0:a] + text[c+len(end):len(text)]
        a = find_start()
    return text

""" Format a subset of MediaWiki markup """
def format(text):
    # strip
    text = strip_blocks(text)
    text = text.lstrip()
     
    # format links
    text = int_link_re.sub(int_link_replace, text)
    text = ext_link_re.sub(ext_link_replace, text)
    
    # format by line
    text = text.replace('\t',' ')
    lines = re.split(r'[\s]*\n[\s]*',text)
    formated = []

    # lists handling functions 
    def open_list(list_type, line):
        if list_type == '*': return '<ul>' + line
        if list_type == '#': return '<ol>' + line
        return line
    def close_list(list_type):
        end = len(formated) - 1
        if list_type == '*': formated[end] += '</ul> '
        if list_type == '#': formated[end] += '</ol> '
    last_list = ''

    for line in lines:
        # go through and process line by line
        original = line
        
        # headings
        line = re.sub(r'====+(.*?)====+',       r'<h4>\1</h4> ', line)
        line = re.sub(r'===+(.*?)===+',           r'<h3>\1</h3> ', line)
        line = re.sub(r'==+(.*?)==+',             r'<h2>\1</h2> ', line)

        # lists
        match = re.match('^[\s]*([\*#]+)(.*)', line)
        if match == None: cur_list = ''
        else: 
            cur_list = match.group(1)
            line = match.group(2)
        if cur_list != '' or last_list != '':
            length = max(len(cur_list), len(last_list))
            if cur_list != '': line = '<li>%s</li>' % line
            
            for i in range(length):
                cur = cur_list[i:i+1]
                last = last_list[i:i+1]
                
                if cur == last: continue
                if last == '': line = open_list(cur, line)
                if cur == '':  close_list(last)
                if cur != '' and last != '' and cur != last:
                    close_list(last)
                    line = open_list(cur, line)                    
        last_list = cur_list

        # definitions
        line = re.sub(r'^[\s]*;(.*?):(.*)',     r'<dt>\1</dt><dd>\2</dd> ', line)
        line = re.sub(r'^[\s]*:(.*)',           r'<dd>\1</dd> ', line)
        line = re.sub(r'^[\s]*;(.*)',           r'<dt>\1</dt> ', line)
        
        # format to paragraph
        if line == original and line.strip() != '':
            line = '<p>%s</p> ' % line
        
        # bold/italic
        line = re.sub(r"'''''+(.*?)'''''+",       r'<b><i>\1</i></b>', line)
        line = re.sub(r"'''+(.*?)'''+",           r'<b>\1</b>', line)
        line = re.sub(r"''+(.*?)''+",             r'<i>\1</i>', line)
        
        formated.append(line)
    
    # strip any left over formatting
    text = ' '.join(formated)
    text = extra_re.sub('', text)
    return text

    

""" unit testing """
def test_link():
    def test(text):
        print "%s    >    %s" % (text, int_link_re.sub(int_link_replace, text))
    
    test('[[hi mom]]')
    test('[[hi mom|mom]]')
    test("[http://www.ars-antiqua-austria.com/musik/bononcini.mp3[[Antonio Maria Bononcini]]: ''Vorrei pupille belle'', sung by Radu Marian]")
    
    def test(text):
        print "%s    >    %s" % (text, ext_link_re.sub(ext_link_replace, text))
    
    test('[http://blah.com]')
    test('[http://blah.com blah]')
    test('[http://blah.com blah blah blah]')
    test("[http://www.ars-antiqua-austria.com/musik/bononcini.mp3[[Antonio Maria Bononcini]]: ''Vorrei pupille belle'', sung by Radu Marian]")
    
    e = int_link_re.sub(int_link_replace, "[http://www.ars-antiqua-austria.com/musik/bononcini.mp3[[Antonio Maria Bononcini]]: ''Vorrei pupille belle'', sung by Radu Marian]")
    e = test(e)
    print e
    
    def test(text):
        print "%s    >    %s" % (text, int_link_re.sub(r'\2', text))
    
    test('[[hi mom]]')
    test('[[hi mom|mom]]')
    
    def test(text):
        print "%s    >    %s" % (text, ext_link_re.sub(r'\2', text))
    
    test('[http://blah.com]')
    test('[http://blah.com blah]')
    test('[http://blah.com blah blah blah]')
    
    
    def test(text):
        print "%s    >    %s" % (text, lang_re.sub(r'\1', text))
    
    test("{{lang|de|Republik Österreich}}")
    test("{{lang|Republik Österreich}}")
    

def test_format():
    def test(text):
        print format(text)
        
    test("'''bold text''' here")
    test("'''bold all text")

def test_lists():
    def test(text):
        print format(text)
    
    print '\n\n'
    test('* single line should not be closed')
    test('* this is a simple list\n ')
    test('# a simple ordered list\n ')
    
    print '\n\n'
    test('*a \n *multi \n    *line  \n  *    list\n')
    
    print '\n\n'
    test('*a \n **multi \n    ***line  \n  *    list\n')
    
    print '\n\n'
    test('*a \n *#mixed \n    *#*line  \n  *    list\n')
    

def test_remove():
    def test(text):
        print remove('[',']',text)
    test('this is a [test]passed test')
    test('[this is a] test')
    test('[this [[is] a] test')
    test('[this [is] a] test [delete [me]] passed')
    test('[this [is] a] test [delete] [delete me [too [please]]]')
    
    def test(text):
        print remove('[',']',text, init = '\[image:')
    test('this is a [image:test]passed test')
    test('[this is a] test')
    test('[image:this [[is] a] test')
    test('[image:this [[is]] a] test [image:delete [me]] passed')
    test('[image:this [is] a] test [delete] [image:delete me [too [please]]]')
    
    f = """{| style='float:right;'
    |-
    |{{JanuaryCalendar}}
    |-
    |{{ThisDateInRecentYears|Month=January|Day=16}}
    |}
    {{Day}}"""
    print remove('{{','}}',f) + '\n\n'
    print remove('{|','|}',f)
    
    
   
if __name__ == "__main__":
    test_remove()
    test_lists()
    test_format()
    test_link()
